package fetch;

import com.mongodb.DBObject;
import db.mongo.MongoDao;
import entity.NeedFetchUrl;
import exception.CookieExpiredException;
import exception.NullPageContentException;
import module.index.IndexSolr;
import org.apache.http.*;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import regex.*;

import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.util.*;

/**
 * @fileName: Fetcher
 * @author: Han
 * @date: 2018/5/29 13:12
 * @dscription: 模拟登陆新浪舆情通, 抓取指定数据
 */
public class Fetcher {

    private String username;
    private String password;
    private String yuqingtongInitCookie;
    private String weiboInitCookie;
    private boolean isManualCookie;

    public Fetcher() {
    }

    /**
     * 手动输入cookie
     *
     * @param yuqingtongInitCookie cookie
     */
    Fetcher(String yuqingtongInitCookie) {
        isManualCookie = true;
        this.yuqingtongInitCookie = yuqingtongInitCookie;
        getWeiboInitCookie();
    }

    /**
     * 输入账号名密码,初始化时登录
     *
     * @param username 用户名
     * @param password 密码
     * @param proxy    代理
     */
    private Fetcher(String username, String password, HttpHost proxy) {
        this.username = username;
        this.password = password;
        //登录,返回cookie
        yuqingtongInitCookie = login(username, password, proxy);
        getWeiboInitCookie();
    }

    public static void main(String... args) {
        //获取代理
        HttpHost proxy = getProxy();
        //如果没有取到代理,抛出异常
        if (Objects.isNull(proxy)) {
            throw new NullPointerException("没有代理对象");
        }
        //Fetcher fetcher = new Fetcher("ssqjyj03", "hz234561", proxy);

        Fetcher fetcher = new Fetcher("yqt365=userSId_yqt365_ssqjyj03_32507");
        List<String> sourceTypeList = Collections.singletonList("全部");
        List<Map<String, String>> infoList = fetcher.searchInfo("氤氲之息", true, 3,
                sourceTypeList, "10days", 1, proxy);
        for (int i = infoList.size() - 2; i >= 0; i--) {
            infoList.remove(i);
        }
        infoList = fetcher.fillOriginalContent(infoList, proxy);
        //IndexSolr indexSolr = IndexSolr.getIndexSolrInstance();
        for (Map<String, String> map : infoList) {
            System.out.println(map);
            /*long id = System.currentTimeMillis();
            String url = map.get("原文链接");
            String title = map.get("标题");
            String content = map.get("原文");
            String source = map.get("来源");
            String digest = map.get("摘要");
            String time = map.get("time").replace("年", "").replace("月", "")
                    .replace("日", "").replace(" ", "").replace(":", "");
            NeedFetchUrl needFetchUrl = new NeedFetchUrl(id, url, title, content, source, time, digest, 1);
            boolean isSucceeded = indexSolr.sendToIndex(needFetchUrl);
            if (isSucceeded) {
                System.out.println("索引添加成功..." + needFetchUrl.getUrl());
            } else {
                System.out.println("索引添加失败..." + needFetchUrl.getUrl());
            }*/
        }
        /*try {
            System.out.println(fetcher.getTiebaContent("http://tieba.baidu.com/p/5733901108",proxy));
        }catch (Exception e){
            e.printStackTrace();
        }*/
    }

    /**
     * 从指定库中去除代理ip和端口
     *
     * @return 代理对象
     */
    private static HttpHost getProxy() {

        HttpHost proxy = null;

        Random random = new Random();
        //获取HttpHost,如果获取失败,重复指定次数,超过则退出程序
        int maxRepeatCount = 10;
        int repeatCount = 0;
        while (repeatCount++ <= maxRepeatCount) {
            try {
                //随机查找记录
                int flagInt = random.nextInt(6) + 1;
                String[] keys = {"flag"};
                Object[] values = {flagInt};
                DBObject dbObject = MongoDao.getMongoDaoInstance().find("siteHost", "proxy", 1, keys, values).get(0);
                String ip = (String) dbObject.get("ip");
                int port = (Integer) dbObject.get("port");
                proxy = new HttpHost(ip, port);
                break;
            } catch (Exception e) {
                if (repeatCount <= maxRepeatCount) {
                    //如果失败,等待1秒重试
                    try {
                        Thread.sleep(1000 * 3);
                    } catch (InterruptedException ignore) {
                    }
                    System.err.println("获取代理ip失败,重试第" + repeatCount + "次");
                } else {
                    System.err.println("获取代理ip超时");
                    proxy = null;
                }
            }
        }

        return proxy;
    }

    /**
     * 使用各种关键字获取全部解析后的舆情通数据
     *
     * @param keyword        关键字
     * @param isPrecised     是否精确搜索
     * @param isSensitive    是否搜索敏感信息.0:全部,3:敏感,4:非敏感
     * @param sourceTypeList 来源类型
     * @param period         时间范围
     *                       today:今天
     *                       24h:24小时
     *                       2days:两天
     *                       3days:三天
     *                       week:一周
     *                       10days:十天
     * @param proxy          代理
     * @return 目标数据
     */
    public List<Map<String, String>> searchAllInfo(String keyword, boolean isPrecised, int isSensitive,
                                                   List<String> sourceTypeList, String period, HttpHost proxy) {

        String pageContent = search(keyword, isPrecised, isSensitive, "", "", sourceTypeList, period, 1, proxy);
        int totalPages = YuqingtongRegex.parseTotalPages(pageContent);
        List<Map<String, String>> allInfoList = YuqingtongRegex.parseContent(pageContent);
        for (int i = 2; i <= totalPages; i++) {
            pageContent = search(keyword, isPrecised, isSensitive, "", "", sourceTypeList, period, i, proxy);
            List<Map<String, String>> tempList = YuqingtongRegex.parseContent(pageContent);
            allInfoList.addAll(tempList);
        }

        return allInfoList;
    }

    /**
     * 使用各种关键字获取指定页码解析后的舆情通数据
     *
     * @param keyword        关键字
     * @param isPrecised     是否精确搜索
     * @param isSensitive    是否搜索敏感信息.0:全部,3:敏感,4:非敏感
     * @param sourceTypeList 来源类型
     * @param period         时间范围
     *                       today:今天
     *                       24h:24小时
     *                       2days:两天
     *                       3days:三天
     *                       week:一周
     *                       10days:十天
     * @param pageNum        页码
     * @param proxy          代理
     * @return 目标数据
     */
    public List<Map<String, String>> searchInfo(String keyword, boolean isPrecised, int isSensitive,
                                                List<String> sourceTypeList, String period, int pageNum, HttpHost proxy) {

        String pageContent = search(keyword, isPrecised, isSensitive, "2018-05-30 00:00:00", "2018-06-08 23:59:59", sourceTypeList, period, pageNum, proxy);
        return YuqingtongRegex.parseContent(pageContent);
    }

    /**
     * 填充原文内容
     *
     * @param list  list
     * @param proxy 代理
     * @return 返回填充后的list
     */
    public List<Map<String, String>> fillOriginalContent(List<Map<String, String>> list, HttpHost proxy) {

        for (Map<String, String> map : list) {
            String sourceSite = map.get("来源");
            String originalUrl = map.get("原文链接");
            String detailId = map.get("舆情通全文地址ID");
            String originalContent = "";
            try {
                if ("百度贴吧".equals(sourceSite)) {
                    originalContent = getTiebaContent(originalUrl, proxy);
                } else if ("新浪微博".equals(sourceSite)) {
                    originalContent = getWeiboContentByGet(originalUrl, proxy);
                } else {
                    originalContent = getCommonSiteContent(originalUrl, proxy);
                }
                if (Objects.isNull(originalContent) || originalContent.isEmpty()) {
                    throw new NullPageContentException("没有获取到正文");
                }
            } catch (Exception e) {
                //获取舆情通检索的页面内容
                String content = getYuqingtongDetailByGet(detailId, proxy);
                //解析页面内容
                originalContent = YuqingtongRegex.parseOriginalContent(content);
            } finally {
                originalContent = originalContent.replaceAll("[\\s]*", "").replace("&nbsp;", "");
                map.put("原文", originalContent);
            }
        }

        return list;
    }

    /**
     * 使用用户名密码,代理登录获取cookie
     *
     * @param username 用户名
     * @param password 密码
     * @param proxy    代理
     * @return 登录cookie
     */
    private String login(String username, String password, HttpHost proxy) {

        //请求头
        Header[] headers = {
                new BasicHeader("Host", "www.yqt365.com"),
                new BasicHeader("Connection", "keep-alive"),
                new BasicHeader("Cache-Control", "max-age=0"),
                new BasicHeader("Origin", "https://www.yqt365.com"),
                new BasicHeader("Upgrade-Insecure-Requests", "1"),
                new BasicHeader("Content-Type", "application/x-www-form-urlencoded"),
                new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36"),
                new BasicHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"),
                new BasicHeader("Referer", "https://www.yqt365.com/logout.action"),
                new BasicHeader("Accept-Encoding", "gzip, deflate, br"),
                new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9"),
                new BasicHeader("Cookie", "UM_distinctid=163aac833761b5-0b292506a6ef6f-3a61430c-1fa400-163aac833774c"),
        };
        //格式数据
        NameValuePair[] formDatas = {
                new BasicNameValuePair("username", username),
                new BasicNameValuePair("password", password),
                new BasicNameValuePair("vcode", ""),
                new BasicNameValuePair("_ran", ""),
        };
        List<NameValuePair> nameValuePairList = Arrays.asList(formDatas);
        //目标url
        String urlStr = "https://www.yqt365.com/login.action";
        int maxRepeatCount = 2;
        int repeatCount = 0;
        String cookie;
        while (repeatCount++ <= maxRepeatCount) {
            try {
                Map<String, List<String>> map = getInfoByPost(urlStr, headers, nameValuePairList, proxy);
                List<String> cookieList = map.get("Set-Cookie");
                //如果没有cookie,返回
                if (cookieList.isEmpty()) {
                    return "";
                }
                cookie = cookieList.get(0);
                //定位开头
                int tempFlag = cookie.indexOf("yqt365=");
                //为了确定结尾的";"截取字符串
                cookie = cookie.substring(tempFlag);
                //确定结尾位置
                tempFlag = cookie.indexOf(";");
                //截取最终cookies
                cookie = cookie.substring(0, tempFlag);
                return cookie;
            } catch (SocketException | SocketTimeoutException | ConnectTimeoutException e) {
                e.printStackTrace();
                if (repeatCount <= maxRepeatCount) {
                    System.err.println("连接超时,再次获取代理重试第" + repeatCount + "次");
                    //重新获取代理
                    proxy = getProxy();
                } else {
                    System.err.println("连接超时,登录失败");
                    break;
                }
            } catch (Exception e) {
                e.printStackTrace();
                System.err.println("未知错误" + e.getMessage());
                break;
            }
        }
        return "";
    }

    /**
     * 使用关键词,页码,cookie,代理获取页面内容
     *
     * @param keyword        关键字
     * @param isPrecised     是否精确搜索
     * @param isSensitive    是否搜索敏感信息.0:全部,3:敏感,4:非敏感
     * @param sourceTypeList 来源类型
     * @param period         时间范围
     *                       today:今天
     *                       24h:24小时
     *                       2days:两天
     *                       3days:三天
     *                       week:一周
     *                       10days:十天
     * @param pageNum        页码
     * @param proxy          代理
     * @return 目标数据
     */
    private String search(String keyword, boolean isPrecised, int isSensitive, String startTime, String endTime,
                          List<String> sourceTypeList, String period, int pageNum, HttpHost proxy) {
        //请求头
        Header[] headers = {
                new BasicHeader("Host", "www.yqt365.com"),
                new BasicHeader("Connection", "keep-alive"),
                new BasicHeader("Accept", "*/*"),
                new BasicHeader("Origin", "https://www.yqt365.com"),
                new BasicHeader("X-Requested-With", "XMLHttpRequest"),
                new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36"),
                new BasicHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"),
                new BasicHeader("Referer", "https://www.yqt365.com/searchList.action"),
                new BasicHeader("Accept-Encoding", "gzip, deflate, br"),
                new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9"),
                new BasicHeader("Cookie", yuqingtongInitCookie)
        };
        String sourceType;
        if (sourceTypeList.contains("全部")) {
            //如果查询全部,直接指定来源类型
            sourceType = "1";
        } else {
            //获得来源类型
            Map<String, Integer> sourceTypeMap = new HashMap<>(11);
            sourceTypeMap.put("网站", 4);
            sourceTypeMap.put("微信", 5);
            sourceTypeMap.put("微博", 2);
            sourceTypeMap.put("客户端", 6);
            sourceTypeMap.put("论坛", 3);
            sourceTypeMap.put("新闻", 7);
            sourceTypeMap.put("博客", 8);
            sourceTypeMap.put("政务", 9);
            sourceTypeMap.put("报刊", 10);
            sourceTypeMap.put("外媒", 11);
            sourceTypeMap.put("视频", 12);
            //转换来源类型为指定格式
            StringBuilder sourceTypeSB = new StringBuilder();
            for (String str : sourceTypeList) {
                int code = sourceTypeMap.get(str);
                sourceTypeSB.append(code).append(",");
            }
            sourceType = sourceTypeSB.substring(0, sourceTypeSB.length() - 1);
        }
        //设置是否精确搜索
        String precise;
        if (isPrecised) {
            precise = "1";
        } else {
            precise = "2";
        }
        //格式数据
        NameValuePair[] formDatas = {
                new BasicNameValuePair("checkedIds", ""),
                new BasicNameValuePair("selectedId", ""),
                new BasicNameValuePair("searchFrom", "2"),
                new BasicNameValuePair("kw.id", ""),
                new BasicNameValuePair("searchKeyword", keyword),
                new BasicNameValuePair("searchPipeiType", precise),
                new BasicNameValuePair("pinglunShow", ""),
                new BasicNameValuePair("zhaiyaoShow", "1"),
                new BasicNameValuePair("duplicateShow", "0"),
                new BasicNameValuePair("paixu", "2"),
                new BasicNameValuePair("highLightKeywords", ""),
                new BasicNameValuePair("endtime", endTime),
                new BasicNameValuePair("starttime", startTime),
                new BasicNameValuePair("clickFilterOrigina", sourceType),
                new BasicNameValuePair("filterOrigina", sourceType),
                new BasicNameValuePair("clickPaixu", "3"),
                new BasicNameValuePair("clickOtherAttribute", isSensitive + ""),
                new BasicNameValuePair("clickDuplicateShow", "-1"),
                new BasicNameValuePair("otherAttribute", "0"),
                new BasicNameValuePair("fontSizeType", "1"),
                new BasicNameValuePair("clickFontSizeType", ""),
                new BasicNameValuePair("newlstSelect", "1"),
                new BasicNameValuePair("clickNewlstSelect", "-1"),
                new BasicNameValuePair("comblineflg", "2"),
                new BasicNameValuePair("clickComblineflg", "-1"),
                new BasicNameValuePair("clickZhaiyaoShow", ""),
                new BasicNameValuePair("dataParam", "0"),
                new BasicNameValuePair("defaultLen", period),
                new BasicNameValuePair("secondSearchWord", ""),
                new BasicNameValuePair("toolbarSwitch", "0"),
                new BasicNameValuePair("page", pageNum + ""),
                new BasicNameValuePair("total", "0"),
                new BasicNameValuePair("searchType", "2"),
                new BasicNameValuePair("isOnlySearchRootWb", "0"),
                new BasicNameValuePair("isClickOnlySearchRootWb", "-1"),
                new BasicNameValuePair("isCountOriginType", "0"),
        };
        List<NameValuePair> nameValuePairList = Arrays.asList(formDatas);
        //目标url
        String urlStr = "https://www.yqt365.com/searchListMore.action";

        int maxRepeatCount = 2;
        int repeatCount = 0;
        String pageContent = "";
        while (repeatCount++ <= maxRepeatCount) {
            try {
                Map<String, List<String>> map = getInfoByPost(urlStr, headers, nameValuePairList, proxy);
                List<String> pageContentList = map.get("Page-Content");
                //如果没有页面源码,重试
                if (pageContentList.isEmpty()) {
                    continue;
                }
                pageContent = pageContentList.get(0);
                if (!pageContent.contains("查看原文")) {
                    pageContent = "0";
                    throw new CookieExpiredException();
                }
                break;
            } catch (SocketException | SocketTimeoutException | ConnectTimeoutException e) {
                pageContent = "0";
                e.printStackTrace();
                if (repeatCount <= maxRepeatCount) {
                    System.err.println("连接超时,获取代理重试第" + repeatCount + "次");
                    //重新获取代理
                    proxy = getProxy();
                } else {
                    System.err.println("连接超时,搜索失败");
                    break;
                }
            } catch (CookieExpiredException e) {
                e.printStackTrace();
                pageContent = "0";
                if (!isManualCookie) {
                    //登陆过期,重启登录
                    yuqingtongInitCookie = login(username, password, proxy);
                    headers[10] = new BasicHeader("Cookie", yuqingtongInitCookie);
                } else {
                    System.err.println("Cookie已过期,请重新输入");
                    break;
                }
            } catch (Exception e) {
                e.printStackTrace();
                System.err.println("连接" + urlStr + "超时");
            }
        }

        return pageContent.replace("<font color=\"red\">", "").replace("</font>", "");
    }

    /**
     * 通过post方式获取HttPResponse
     *
     * @param urlStr            url
     * @param headers           请求头
     * @param nameValuePairList 表单数据
     * @param proxy             代理
     * @return 对象
     * @throws Exception 抛出异常
     */
    private Map<String, List<String>> getInfoByPost(String urlStr, Header[] headers, List<NameValuePair>
            nameValuePairList, HttpHost proxy) throws Exception {
        Map<String, List<String>> infoMap = new HashMap<>(5);
        //建立HttpClient对象
        HttpClient httpClient = HttpClients.createDefault();
        //建立POST请求
        HttpPost post = null;
        try {
            post = new HttpPost(urlStr);
            //配置代理
            RequestConfig config = RequestConfig.custom().setProxy(proxy).setConnectTimeout(1000 * 3)
                    .setSocketTimeout(1000 * 3).setConnectionRequestTimeout(1000 * 3).build();
            post.setConfig(config);
            post.setHeaders(headers);
            HttpEntity formData = new UrlEncodedFormEntity(nameValuePairList, Consts.UTF_8);
            post.setEntity(formData);
            //发送POST请求
            HttpResponse response = httpClient.execute(post);
            //获取状态码
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode == HttpStatus.SC_OK) {
                HttpEntity entity = response.getEntity();
                // 获取字节数组
                byte[] content = EntityUtils.toByteArray(entity);
                //页面内容
                String pageContent = new String(content, "utf-8");
                List<String> pageContentList = new ArrayList<>();
                pageContentList.add(pageContent);
                infoMap.put("Page-Content", pageContentList);
            } else if (statusCode == HttpStatus.SC_MULTIPLE_CHOICES || statusCode == HttpStatus.SC_MOVED_PERMANENTLY
                    || statusCode == HttpStatus.SC_MOVED_TEMPORARILY || statusCode == HttpStatus.SC_SEE_OTHER
                    || statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_USE_PROXY
                    || statusCode == HttpStatus.SC_TEMPORARY_REDIRECT) {
                //获取响应头,放入Map中
                Header[] responseAllHeaders = response.getAllHeaders();
                for (Header header : responseAllHeaders) {
                    String name = header.getName();
                    //检查是否已经存在键对应的List,如果不存在,创建
                    List<String> tempList = infoMap.get(name);
                    if (Objects.isNull(tempList)) {
                        tempList = new ArrayList<>();
                    }
                    String value = header.getValue();
                    tempList.add(value);
                    infoMap.put(name, tempList);
                }
            }
        } catch (Exception e) {
            //出现异常,先关闭连接再抛出
            Objects.requireNonNull(post).abort();
            throw e;
        } finally {
            Objects.requireNonNull(post).abort();
        }

        return infoMap;
    }

    /**
     * 使用代理获得页面内容
     *
     * @param urlStr  目的地址
     * @param headers 请求头
     * @param proxy   代理
     * @return 页面内容
     * @throws Exception 向上抛出异常
     */
    private Map<String, List<String>> getInfoByGet(String urlStr, Header[] headers, HttpHost proxy) throws Exception {

        Map<String, List<String>> map = new LinkedHashMap<>();

        HttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = null;
        try {
            httpGet = new HttpGet(urlStr);
            //配置代理
            RequestConfig config = RequestConfig.custom().setProxy(proxy).setConnectTimeout(1000 * 3)
                    .setSocketTimeout(1000 * 3).setConnectionRequestTimeout(1000 * 3).build();
            httpGet.setConfig(config);
            httpGet.setHeaders(headers);
            HttpResponse response = httpClient.execute(httpGet);

            //放入状态码
            int statusCode = response.getStatusLine().getStatusCode();
            map.put("Status-Code", Collections.singletonList(statusCode + ""));
            //放入页面内容
            Header[] responseAllHeaders = response.getAllHeaders();
            String charset = "utf-8";
            for (Header header : responseAllHeaders) {
                String name = header.getName();
                //检查是否已经存在键对应的List,如果不存在,创建
                List<String> tempList = map.get(name);
                if (Objects.isNull(tempList)) {
                    tempList = new ArrayList<>();
                }
                String value = header.getValue();
                tempList.add(value);
                map.put(name, tempList);
                if (value.contains("charset")) {
                    int start = value.indexOf("charset=") + 8;
                    charset = value.substring(start);
                }
            }

            HttpEntity entity = response.getEntity();
            //页面内容
            String pageContent = EntityUtils.toString(entity, charset);
            map.put("Page-Content", Collections.singletonList(pageContent));
        } catch (Exception e) {
            Objects.requireNonNull(httpGet).abort();
            throw e;
        } finally {
            Objects.requireNonNull(httpGet).abort();
        }

        return map;
    }

    /**
     * 获取微博的初始化cookie
     */
    private void getWeiboInitCookie() {

        Random random = new Random();

        weiboInitCookie = "SUB=" + (random.nextInt(9) + 1);
        /*//获取初始化cookie
        String initUrl = "https://passport.weibo" +
                ".com/visitor/visitor?a=incarnate&t=c5pOG3Sk4IYqEsguWtbGi8sNPuXGaWXZyl5bTx4kiIQ%3D&w=2&c=095&gc=&cb=cross_domain&from=weibo&_rand=0.9902541622453138";
        Header[] headers = {
                new BasicHeader("Host", "passport.weibo.com"),
                new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36"),
        };

        int maxRepeatCount = 2;
        int repeatCount = 0;
        while (repeatCount++ <= maxRepeatCount) {
            try {
                //获取微博初始化cookie
                Map<String, List<String>> tempMap = getInfoByGet(initUrl, headers, proxy);
                List<String> cookieList = tempMap.get("Set-Cookie");
                StringBuilder tempCookieSB = new StringBuilder();
                for (String tempCookie : cookieList) {
                    tempCookieSB.append(tempCookie);
                }
                weiboInitCookie = tempCookieSB.toString();
                break;
            } catch (SocketException | SocketTimeoutException | ConnectTimeoutException e) {
                if (repeatCount <= maxRepeatCount) {
                    System.err.println("连接超时,获取代理重试第" + repeatCount + "次");
                    //重新获取代理
                    proxy = getProxy();
                } else {
                    System.err.println("连接超时,使用舆情通抓取的页面");
                    break;
                }
            } catch (Exception e) {
                break;
            }
        }*/
    }

    /**
     * 获取微博url的页面内容,如果获取失败,则使用舆情通抓取的页面
     *
     * @param urlStr 微博url
     * @param proxy  代理
     * @return 页面内容
     */
    private String getWeiboContentByGet(String urlStr, HttpHost proxy) throws Exception {

        String pageContent = "";

        //设置请求头
        Header[] headers = {
                new BasicHeader("Host", "weibo.com"),
                new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, " +
                        "like Gecko) Chrome/67.0.3396.62 Safari/537.36"),
                new BasicHeader("Cookie", weiboInitCookie)
        };
        //获取目标url页面内容
        int maxRepeatCount = 2;
        int repeatCount = 0;
        while (repeatCount++ <= maxRepeatCount) {
            try {
                Map<String, List<String>> tempMap = getInfoByGet(urlStr, headers, proxy);
                String statusCode = tempMap.get("Status-Code").get(0);
                char firstChar = statusCode.charAt(0);
                if (firstChar == '3' || firstChar == '4' || firstChar == '5') {
                    throw new NullPageContentException("状态码:" + statusCode + ",没有页面内容");
                }
                pageContent = tempMap.get("Page-Content").toString();
                break;
            } catch (SocketException | SocketTimeoutException | ConnectTimeoutException e) {
                if (repeatCount <= maxRepeatCount) {
                    System.err.println("连接超时,获取代理重试第" + repeatCount + "次");
                    //重新获取代理
                    proxy = getProxy();
                    getWeiboInitCookie();
                } else {
                    throw new NullPageContentException("连接超时,使用舆情通抓取的页面:" + urlStr);
                }
            } catch (Exception e) {
                throw new NullPageContentException("获取微博内容失败,使用舆情通抓取的页面:" + urlStr);
            }
        }
        //解析页面内容
        pageContent = WeiboRegex.parseWeibo(pageContent);
        if (pageContent.isEmpty()) {
            throw new NullPageContentException("没有检索到原链接内容:" + urlStr);
        }

        return pageContent;
    }

    /**
     * 使用指定id获取页面数据
     *
     * @param id    详情页id
     * @param proxy 代理
     * @return 页面数据
     */
    private String getYuqingtongDetailByGet(String id, HttpHost proxy) {

        String pageContent = "";

        Header[] headers = {
                new BasicHeader("Host", "www.yqt365.com"),
                new BasicHeader("Connection", "keep-alive"),
                new BasicHeader("Upgrade-Insecure-Requests", "1"),
                new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36"),
                new BasicHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"),
                new BasicHeader("Accept-Encoding", "gzip, deflate, br"),
                new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9"),
                new BasicHeader("Cookie", yuqingtongInitCookie)
        };

        String urlStr = "https://www.yqt365.com/goArticleDetailView.action?checkedIds=," + id
                + ",%2525E6%2525B0%2525A4%2525E6%2525B0%2525B2%2525E4%2525B9%25258B%2525E6%252581%2525AF,2,1,0,,,-1,0";
        int maxRepeatCount = 2;
        int repeatCount = 0;
        while (repeatCount++ <= maxRepeatCount) {
            try {
                Map<String, List<String>> map = getInfoByGet(urlStr, headers, proxy);
                pageContent = map.get("Page-Content").toString();
                break;
            } catch (SocketException | SocketTimeoutException | ConnectTimeoutException e) {
                if (repeatCount <= maxRepeatCount) {
                    System.err.println("连接超时,获取代理重试第" + repeatCount + "次");
                    //重新获取代理
                    proxy = getProxy();
                } else {
                    System.err.println("连接超时,使用舆情通抓取的页面");
                    break;
                }
            } catch (Exception e) {
                System.err.println("连接错误,使用舆情通抓取的页面:" + e.getMessage());
                break;
            }
        }

        return pageContent;
    }

    /**
     * 解析贴吧内容,如果有多页,检索每一页
     *
     * @param urlStr 帖子地址
     * @param proxy  代理
     * @return 帖子内容
     */
    private String getTiebaContent(String urlStr, HttpHost proxy) throws Exception {

        Header[] headers = {
                new BasicHeader("Host", "tieba.baidu.com"),
                new BasicHeader("Connection", "keep-alive"),
                new BasicHeader("Upgrade-Insecure-Requests", "1"),
                new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36"),
                new BasicHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"),
                new BasicHeader("Accept-Encoding", "gzip, deflate, br"),
                new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9")
        };
        StringBuilder tiebaContentSB = new StringBuilder();
        String pageContent = "";
        int maxRepeatCount = 2;
        int repeatCount = 0;
        while (repeatCount++ <= maxRepeatCount) {
            try {
                Map<String, List<String>> map = getInfoByGet(urlStr + "?pn=" + 1, headers, proxy);
                pageContent = map.get("Page-Content").toString();
                break;
            } catch (SocketException | SocketTimeoutException | ConnectTimeoutException e) {
                if (repeatCount <= maxRepeatCount) {
                    System.err.println("连接超时,获取代理重试第" + repeatCount + "次");
                    //重新获取代理
                    proxy = getProxy();
                } else {
                    throw new NullPageContentException("帖子:" + urlStr + "获取失败,使用舆情通抓取页面");
                }
            } catch (Exception e) {
                throw new NullPageContentException("帖子:" + urlStr + "获取失败,使用舆情通抓取页面");
            }
        }
        //使用正则表达式匹配出总页码
        int pagination = TiebaRegex.parsePagination(pageContent);
        //第一页内容
        tiebaContentSB.append(TiebaRegex.parseContent(pageContent));
        //多页内容
        for (int i = 2; i <= pagination; i++) {
            String url = urlStr + "?pn=" + i;
            maxRepeatCount = 2;
            repeatCount = 0;
            while (repeatCount++ <= maxRepeatCount) {
                try {
                    Map<String, List<String>> map = getInfoByGet(url, headers, proxy);
                    pageContent = map.get("Page-Content").toString();
                    tiebaContentSB.append(TiebaRegex.parseContent(pageContent));
                    break;
                } catch (SocketException | SocketTimeoutException | ConnectTimeoutException e) {
                    if (repeatCount <= maxRepeatCount) {
                        System.err.println("连接超时, 获取代理重试第" + repeatCount + "次");
                        //重新获取代理
                        proxy = getProxy();
                    } else {
                        System.err.println(url + "没有抓取到");
                        break;
                    }
                } catch (Exception e) {
                    System.err.println(url + "没有抓取到");
                    break;
                }
            }
        }

        return tiebaContentSB.toString();
    }

    /**
     * 获取普通站点内容
     *
     * @param urlStr 目标url
     * @param proxy  代理
     * @return 站点内容
     * @throws Exception 抛出异常
     */
    private String getCommonSiteContent(String urlStr, HttpHost proxy) throws Exception {

        String hostStr = Regex.matchString(urlStr, "(?<=http[s]{0,1}://)[\\s\\S]*?[^/](?=/)");
        Header[] headers = {
                new BasicHeader("Host", hostStr),
                new BasicHeader("Connection", "keep-alive"),
                new BasicHeader("Upgrade-Insecure-Requests", "1"),
                new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36"),
                new BasicHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"),
                new BasicHeader("Accept-Encoding", "gzip, deflate, br"),
                new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9")
        };
        String pageContent = "";
        int maxRepeatCount = 2;
        int repeatCount = 0;
        while (repeatCount++ <= maxRepeatCount) {
            try {
                Map<String, List<String>> map = getInfoByGet(urlStr, headers, proxy);
                pageContent = map.get("Page-Content").toString();
                break;
            } catch (SocketException | SocketTimeoutException | ConnectTimeoutException e) {
                if (repeatCount <= maxRepeatCount) {
                    System.err.println("连接超时,获取代理重试第" + repeatCount + "次");
                    //重新获取代理
                    proxy = getProxy();
                } else {
                    throw new NullPageContentException(urlStr + "获取失败,使用舆情通抓取页面");
                }
            } catch (Exception e) {
                throw new NullPageContentException(urlStr + "获取失败,使用舆情通抓取页面");
            }
        }

        return CommonRegex.getSignalText(pageContent);
    }
}
